# -*- coding:utf-8 -*-
# @Time:2024/4/1921:01
# @Author:miuzg
# @FileName:new test2.py
# @Software:PyCharm

import os
import requests
from concurrent.futures import ProcessPoolExecutor
from lxml import etree


class Driver:
    def __init__(self):
        self.url = 'https://www.biquie.cc/book/4203/'
        self.headers = {
            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/132.0.0.0 Safari/537.36 Edg/132.0.0.0'
        }

    def get_html(self, url):
        """获取网页"""
        res = requests.get(self.url, headers=self.headers)
        res.encoding = 'utf-8'
        return res.text

    def parse_html(self, html):
        """筛选数据"""
        html = etree.HTML(html)
        # 获取小说标题
        name = html.xpath('//h1/text()')[0]

        # # 创建文件夹
        os.makedirs(name, exist_ok=True)

        # 获取所有章节链接
        chapter_url = html.xpath('//dd/a/@href')

        # 进入对应章节网页
        for url in chapter_url:
            if url == 'javascript:dd_show()':
                continue
            if url == 'https://www.675m.com/book/2505/':
                return
            url = 'https://www.biquie.cc' + url
            # 爬取小说正文
            res = requests.get(url, headers=self.headers)
            html = etree.HTML(res.text)

            # 章节名
            html = etree.HTML(res.text)
            title = html.xpath('//span[@class="title""]/text()')[0]

            # 章节内容
            text = '\n'.join(html.xpath('//div[@id="chaptercontent"]/text()'))

            # 路径为 小说名文件夹\章节名/txt
            with open(fr'{name}\{title}.txt', 'w', encoding='utf-8')as f:
                f.write(text)
                print(fr'{title}保存成功')

    def run(self):
        """执行程序"""
        html = self.get_html(self.url)
        self.parse_html(html)
        # pool.submit(self.parse_html,html)


if __name__ == '__main__':
    pool = ProcessPoolExecutor(4)
    spider = Driver()
    spider.run()
